
import pandas as pd
import os, pickle
import numpy as np
import pymongo

import sys
sys.path.append('../')
import src.IQ as IQ


myclient = pymongo.MongoClient("mongodb://127.0.0.1:27017/admin")
BLE = myclient["BLE"]

def query(collection, filter:dict, addFrameColumn=True):
    df =  pd.DataFrame(list(collection.find(filter)))
    if addFrameColumn:
        df['frame'] = df.apply(lambda x: x['I'] + np.dot(x['Q'],1j), axis=1)
    return df.copy()


from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Dense
from tensorflow.keras.optimizers import Adam
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow.keras.callbacks import EarlyStopping
from sklearn.preprocessing import StandardScaler
from tensorflow.keras.backend import clear_session
from tensorflow.keras.regularizers import l1, l2
import threading
from sklearn.preprocessing import MinMaxScaler





# We will start with a size that is approximately half of the input size
def run(encoding_dim,input_tensor_features,output_tensor_features, plot = False):
    tf.keras.backend.clear_session()
    input_shape = input_tensor_features[0].shape[0]
    output_shape = output_tensor_features[0].shape[0]
    input_data = Input(shape=(input_shape,))
    encoded = Dense(encoding_dim + (input_shape - encoding_dim )//2, activation='relu')(input_data)
    encoded = Dense(encoding_dim + (input_shape - encoding_dim )//4, activation='tanh')(encoded)
    
    encoded = Dense(encoding_dim, activation='linear')(encoded) # kernel_regularizer=l1(0.01)


    decoded = Dense(encoding_dim + (output_shape - encoding_dim )//4, activation='tanh')(encoded) 
    decoded = Dense(encoding_dim + (output_shape - encoding_dim )//2, activation='relu')(decoded)
    decoded = Dense(output_shape, activation='linear')(decoded)

    # This model maps an input to its reconstruction
    autoencoder = Model(input_data, decoded)

    # This model maps an input to its encoded representation
    # encoder = Model(input_data, encoded)

    # Compile the model
    autoencoder.compile(optimizer=Adam(learning_rate=0.001), loss='mean_squared_error')

    # Display the architecture of the autoencoder
    with lock:
        autoencoder.summary()

    # Train the autoencoder
    # We will use new_csv_features as both the input and the target since it's an autoencoder

    early_stopping = EarlyStopping(monitor='val_loss', patience=40, verbose=1)

    # Train the autoencoder and save the history
    history = autoencoder.fit(input_tensor_features, output_tensor_features,
                            epochs=2048,
                            batch_size=16,
                            shuffle=True,
                            validation_split=0.2,
                            callbacks=[early_stopping],
                            verbose=0,
                            )  # Set verbose to 1 for progress output




    best_loss = min(history.history['val_loss']) # this is the raining loss
    best_losses[encoding_dim] = best_loss
    print("latent dim:", encoding_dim, "loss value is: ", best_loss)

    if plot:
        lock.acquire()
        plt.figure(figsize=(10, 2),dpi=80)
        plt.plot(output_tensor_features[199], label='original')
        plt.plot(autoencoder(input_tensor_features)[199], label='reconstructed')
        plt.title('Latnet dim:'+ str(encoding_dim)+", best loss:"+str(best_loss)) 
        plt.legend()
        plt.show()
        plt.close()

        # plt.figure(figsize=(20, 4))
        # for i in range (1,len(autoencoder.layers)-1):
        # # Assuming 'autoencoder' is your trained model and 'layer_index' is the index of the layer you want to analyze
        #     weights = autoencoder.layers[i].get_weights()[0]  # 0 for weights, 1 for biases

        #     # Plotting the weights as a heatmap
            
        #     plt.subplot(1,len(autoencoder.layers)-2,i)
        #     plt.imshow(weights, cmap='viridis', aspect='auto')
        #     plt.colorbar()

        # plt.show()
        # plt.close()

        # plt.figure(figsize=(10, 16))
        # for i in range (1,len(autoencoder.layers)-1):
        # # Assuming 'autoencoder' is your trained model and 'layer_index' is the index of the layer you want to analyze
        #     weights = autoencoder.layers[i].get_weights()[0]  # 0 for weights, 1 for biases
        #     biases = autoencoder.layers[i].get_weights()[1]  # 0 for weights, 1 for biases
        #     # Histogram of the weight values
        #     plt.subplot(2*(len(autoencoder.layers)-2),1,i*2-1)
        #     plt.plot(weights.flatten().tolist(), label='weights')
        #     plt.plot(np.linspace(0,len(weights.flatten()),len(biases.flatten())),biases.flatten().tolist(), label='biases')
        #     plt.legend()
        #     plt.subplot(2*(len(autoencoder.layers)-2),2,i*4-1)
        #     plt.hist(weights.flatten(), bins=50)

            
        #     # Histogram of the biases values
        #     plt.subplot(2*(len(autoencoder.layers)-2),2,i*4)
        #     plt.hist(biases.flatten(), bins=50)

        # plt.show()
        # plt.close()

        lock.release()
    tf.keras.backend.clear_session()
    clear_session()




# Create a StandardScaler object
scaler = StandardScaler()
scaler2 = MinMaxScaler(feature_range=(0, 1))
# Assuming 'new_csv_features' is your data
def normalized(row):
    row = np.array(row)
    return scaler2.fit_transform(scaler.fit_transform(row.reshape(-1, 1)))

def fft_normalized(row):
    row = np.array(row)
    temp = np.fft.fft(row)[0:len(row)//2 + 1]

    amp = normalized(np.abs(temp))
    filtering = amp > np.average(amp)*.05
    angle = normalized(np.angle(temp))

    angle = angle[filtering] 
    angle = np.concatenate([angle,np.zeros(len(row)-len(angle))])
    amp = amp[filtering] 
    amp = np.concatenate([amp,np.zeros(len(row)-len(amp))])
    
    return np.concatenate([amp,angle])

lock = threading.Lock()
batch_size = 1

filtering = {''}
df = query(BLE['onBody'], {'pos':'static','antenna_side':'left'})
min_length = df['frame'].apply(len).min()
df['frame'] = df['frame'].apply(lambda x: x[:2000])
print(len(df['frame'][0]))
print(type(df['frame'][0]))

iq = IQ.IQ(Fc=2439810000+.1e4)

def configCreator(downSampleRate = 1, cutoff = 1e6):
    downSampleRate= max(downSampleRate, 1)
    return {                                      
            # iq.gradient:{},
            iq.unwrapPhase:{},
            iq.phase:{}, 
            # iq.butter:{'Fs': iq.Fs/downSampleRate, "cutoff": cutoff},
            iq.downSample:{'downSampleRate':downSampleRate, "shift": 0},
            iq.demodulate:{'Fs': iq.Fs},
           } 

methods = configCreator(downSampleRate=  10)
df['data'] = iq.apply(methods = methods, frame = df)
####################################################################################
# df['normalized'] = df['predictor'].apply(lambda x: normalized(x))

df['normalized'] = df['data'].apply(lambda x: fft_normalized(x))
input_tensor_features = tf.convert_to_tensor(df['normalized'].tolist())

df['normalized'] = df['data']#.apply(lambda x: normalized(x))
output_tensor_features = tf.convert_to_tensor(df['normalized'].tolist())

data_shape = len(df['data'][0])
#####################################################################################

best_losses = {}
for batch in range(1,int(np.ceil(data_shape/batch_size))):
    try:
        run(batch,output_tensor_features,output_tensor_features,True)
    except:
        continue
    # threads = []
    # for i in range(batch_size * batch+1, min(batch_size * batch+batch_size+1,data_shape)):
    #     run, args=(i,output_tensor_features,output_tensor_features,True, )
    #     threads.append(threading.Thread(target=run, args=(i,output_tensor_features,output_tensor_features,True, )))
    #     threads[-1].start()
    # for t in threads:
    #     t.join()

best_losses = dict(sorted(best_losses.items()))
lossDF = pd.DataFrame(best_losses.values(),index=best_losses.keys())
lossDF.plot()
lossDF.to_csv('loss.csv')
plt.xlabel('Latent dimension')
plt.ylabel('Best loss')
plt.title('dataset: onBody static left')
plt.savefig('res/'+"onBody static left"+'.png')
plt.show()
plt.close()

2023-11-15 21:46:45.044447: I tensorflow/core/util/port.cc:111] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-11-15 21:46:45.067717: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
2023-11-15 21:46:45.067738: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2023-11-15 21:46:45.067754: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2023-11-15 21:46:45.071654: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.

2000
<class 'numpy.ndarray'>

2023-11-15 21:46:53.076839: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:894] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355
2023-11-15 21:46:53.094172: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:894] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355
2023-11-15 21:46:53.094629: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:894] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355
2023-11-15 21:46:53.099765: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:894] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355
2023-11-15 21:46:53.100213: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:894] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355
2023-11-15 21:46:53.100554: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:894] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355
2023-11-15 21:46:53.525973: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:894] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355
2023-11-15 21:46:53.526076: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:894] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355
2023-11-15 21:46:53.526148: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:894] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355
2023-11-15 21:46:53.526205: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1886] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 4620 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3060 Ti, pci bus id: 0000:01:00.0, compute capability: 8.6

Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_1 (InputLayer)        [(None, 200)]             0         
                                                                 
 dense (Dense)               (None, 100)               20100     
                                                                 
 dense_1 (Dense)             (None, 50)                5050      
                                                                 
 dense_2 (Dense)             (None, 1)                 51        
                                                                 
 dense_3 (Dense)             (None, 50)                100       
                                                                 
 dense_4 (Dense)             (None, 100)               5100      
                                                                 
 dense_5 (Dense)             (None, 200)               20200     
                                                                 
=================================================================
Total params: 50601 (197.66 KB)
Trainable params: 50601 (197.66 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________

2023-11-15 21:46:54.852903: I tensorflow/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory
2023-11-15 21:46:55.182067: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x7f2ff401ea20 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
2023-11-15 21:46:55.182091: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA GeForce RTX 3060 Ti, Compute Capability 8.6
2023-11-15 21:46:55.187073: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:269] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.
2023-11-15 21:46:55.357198: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:442] Loaded cuDNN version 8700
2023-11-15 21:46:55.436993: I ./tensorflow/compiler/jit/device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.

Epoch 80: early stopping
latent dim: 1 loss value is:  0.3288440704345703

Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_1 (InputLayer)        [(None, 200)]             0         
                                                                 
 dense (Dense)               (None, 101)               20301     
                                                                 
 dense_1 (Dense)             (None, 51)                5202      
                                                                 
 dense_2 (Dense)             (None, 2)                 104       
                                                                 
 dense_3 (Dense)             (None, 51)                153       
                                                                 
 dense_4 (Dense)             (None, 101)               5252      
                                                                 
 dense_5 (Dense)             (None, 200)               20400     
                                                                 
=================================================================
Total params: 51412 (200.83 KB)
Trainable params: 51412 (200.83 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
Epoch 73: early stopping
latent dim: 2 loss value is:  0.0939439907670021

Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_1 (InputLayer)        [(None, 200)]             0         
                                                                 
 dense (Dense)               (None, 101)               20301     
                                                                 
 dense_1 (Dense)             (None, 52)                5304      
                                                                 
 dense_2 (Dense)             (None, 3)                 159       
                                                                 
 dense_3 (Dense)             (None, 52)                208       
                                                                 
 dense_4 (Dense)             (None, 101)               5353      
                                                                 
 dense_5 (Dense)             (None, 200)               20400     
                                                                 
=================================================================
Total params: 51725 (202.05 KB)
Trainable params: 51725 (202.05 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
Epoch 197: early stopping
latent dim: 3 loss value is:  0.03543655574321747

Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_1 (InputLayer)        [(None, 200)]             0         
                                                                 
 dense (Dense)               (None, 102)               20502     
                                                                 
 dense_1 (Dense)             (None, 53)                5459      
                                                                 
 dense_2 (Dense)             (None, 4)                 216       
                                                                 
 dense_3 (Dense)             (None, 53)                265       
                                                                 
 dense_4 (Dense)             (None, 102)               5508      
                                                                 
 dense_5 (Dense)             (None, 200)               20600     
                                                                 
=================================================================
Total params: 52550 (205.27 KB)
Trainable params: 52550 (205.27 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
Epoch 81: early stopping
latent dim: 4 loss value is:  0.06580591201782227

Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_1 (InputLayer)        [(None, 200)]             0         
                                                                 
 dense (Dense)               (None, 102)               20502     
                                                                 
 dense_1 (Dense)             (None, 53)                5459      
                                                                 
 dense_2 (Dense)             (None, 5)                 270       
                                                                 
 dense_3 (Dense)             (None, 53)                318       
                                                                 
 dense_4 (Dense)             (None, 102)               5508      
                                                                 
 dense_5 (Dense)             (None, 200)               20600     
                                                                 
=================================================================
Total params: 52657 (205.69 KB)
Trainable params: 52657 (205.69 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
Epoch 112: early stopping
latent dim: 5 loss value is:  0.040589649230241776


len(df['I'][0])


import numpy as np
import matplotlib.pyplot as plt

row =np.random.rand(10) 

filename = files[3]
print(filename)
f = open(dataDir+filename, 'rb')
data_tuples = pickle.load(f)
f.close()

df = pd.DataFrame(data_tuples, columns=['predictor', 'label'])

def fft_normalized(row):
    temp = np.fft.fft(row)[0:len(row)//2 + 1]
    amp = normalized(np.abs(temp))
    filtering = amp > np.max(amp)*0.05 

    angle = normalized(np.angle(temp))
    angle = angle[filtering]
    amp = amp[filtering]

    return np.concatenate([amp,angle])


df['normalized'] = df['predictor'].apply(lambda x: fft_normalized(x))
tf.convert_to_tensor( df['normalized'].to_list())


# best_losses = dict(sorted(best_losses.items()))
# lossDF = pd.DataFrame(best_losses.values(),index=best_losses.keys())

def read_loss(file :str | os.DirEntry):
    if type(file) is str:
        if file.endswith(".csv"):
            lossDF = pd.read_csv(file,index_col=0)
            return lossDF
    else:
        if file.name.endswith(".csv"):
            lossDF = pd.read_csv(file.path,index_col=0)
            return lossDF

def plot_loss(file):
        read_loss(file)
        lossDF.plot()
        plt.xlabel('Latent dimension')
        plt.ylabel('Best loss')
        plt.title('dataset: '+file.name)
        plt.savefig('res/'+file.name+'.png')
        plt.show()
        plt.close()

# for file in os.scandir('res/RAE-_l2-0.001/'):
df = read_loss("loss_dataset1.pkl.csv")
import scipy.signal as signal

def smooth(x,window_len=11,window='hanning'):
    if x.ndim != 1:
        raise ValueError("smooth only accepts 1 dimension arrays.")
    if x.size < window_len:
        raise ValueError("Input vector needs to be bigger than window size.")
    if window_len<3:
        return x
    if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
        raise ValueError(f"Window is on of '{window}'")
    s=np.r_[x[window_len-1:0:-1],x,x[-2:-window_len-1:-1]]
    #print(len(s))
    if window == 'flat': #moving average
        w=np.ones(window_len,'d')
    else:
        w=eval(f'np.{window}({window_len})')
    y=np.convolve(w/w.sum(),s,mode='valid')
    return y

plt.plot(smooth(df['0'].to_numpy(),window_len=5, window='hamming'))